# !/bin/bash -e

source IP.conf

echo "start deploy a single mon's cluster ..."
if [ ! -d "/etc/ceph" ]; then
	sudo mkdir /etc/ceph
fi

FSID=`uuidgen`
hostname=$(hostname -s)
ip_address='10.10.9.31'
public_network='10.10.9.0/24'

echo "[global]
fsid = $FSID
mon initial members = $hostname
mon host = $ip_address
public network = $public_network
osd journal size = 1024
osd pool default size = 3
osd pool default min size = 2
osd pool default pg num = 128
osd pool default pgp num = 128
osd crush chooseleaf type = 1
mon allow pool delete = true
# mon max pg per osd = 1000
" | sudo tee -a /etc/ceph/ceph.conf

# mon
echo "[$hostname] start ceph-mon in $hostname ..."
sudo ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
sudo ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
sudo mkdir -p /var/lib/ceph/bootstrap-osd
sudo ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'
sudo ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
sudo ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
sudo chown $USER:$USER /tmp/ceph.mon.keyring
sudo chown -R $USER:$USER /var/lib/ceph
sudo chown -R $USER:$USER /etc/ceph/
sudo monmaptool --create --add $hostname $ip_address --fsid $FSID /tmp/monmap --clobber
sudo -u $USER mkdir -p /var/lib/ceph/mon/ceph-$hostname
sudo -u $USER ceph-mon --mkfs -i $hostname --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
sudo ceph-mon -i $hostname
echo "[$hostname] finish start ceph-mon in $hostname"

# deal warning
sudo ceph config set mon auth_allow_insecure_global_id_reclaim false
sudo ceph mon enable-msgr2

# mgr
echo "[$hostname] start ceph-mgr in $hostname ..."
sudo mkdir -p /var/lib/ceph/mgr/ceph-$hostname
sudo ceph auth get-or-create mgr.$hostname mon 'allow profile mgr' osd 'allow *' mds 'allow *' | sudo tee -a /var/lib/ceph/mgr/ceph-$hostname/keyring 
sudo ceph-mgr -i $hostname
echo "[$hostname] finish start ceph-mgr in $hostname"

# osd
# create osd.disk
# 在node1上创建osd0
# sudo dd if=/dev/zero of=/var/local/osd0.img bs=4M count=4096 oflag=direct
# sudo sgdisk -g --clear /var/local/osd0.img
# sudo vgcreate -y ceph-osds $(sudo losetup --show -f /var/local/osd0.img)
# sudo lvcreate -l 100%VG -n osd0 ceph-osds

# sudo sgdisk -g --clear /var/local/osd1.img
# sudo vgcreate -y ceph-osds $(sudo losetup --show -f /var/local/osd1.img)
# sudo lvcreate -l 100%VG -n osd1 ceph-osds

# sudo sgdisk -g --clear /var/local/osd2.img
# sudo vgcreate -y ceph-osds $(sudo losetup --show -f /var/local/osd2.img)
# sudo lvcreate -l 100%VG -n osd2 ceph-osds

# start ceph-osd
echo "[$hostname] start ceph-osd in $hostname ..."
OSDID=`uuidgen`
OSD_SECRET=$(ceph-authtool --gen-print-key)
OSD_NUM=$(echo "{\"cephx_secret\": \"$OSD_SECRET\"}" | \
   sudo ceph osd new $OSDID -i - \
   -n client.bootstrap-osd -k /var/lib/ceph/bootstrap-osd/ceph.keyring)
echo "[$hostname] get osd.num = $OSD_NUM ..."
sudo mkdir -p /var/lib/ceph/osd/ceph-$OSD_NUM
sudo mkfs.xfs /dev/ceph-osds/osd$OSD_NUM -f > /dev/null
sudo mount /dev/ceph-osds/osd$OSD_NUM /var/lib/ceph/osd/ceph-$OSD_NUM > /dev/null
sudo ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-$OSD_NUM/keyring --name osd.$OSD_NUM --add-key $OSD_SECRET | sed "s/^/[$hostname] &/g"
sudo ceph-osd -i $OSD_NUM --mkfs --osd-uuid $OSDID > /dev/null 2>&1
sudo ceph-osd -i $OSD_NUM > /dev/null 2>&1
echo "[$hostname] finish start ceph-osd.$OSD_NUM in $hostname"

# ceph config set mon mon_max_pg_per_osd 1000
ceph config set mgr mon_max_pg_per_osd 1000
ceph osd pool create .rgw.root 32 32
ceph osd pool create default.rgw.control 32 32
ceph osd pool create default.rgw.log 32 32
ceph osd pool create default.rgw.meta 32 32
ceph osd pool create default.rgw.buckets.data 32 32
ceph osd pool create default.rgw.buckets.index 32 32
ceph osd pool application enable .rgw.root rgw
ceph osd pool application enable default.rgw.control rgw
ceph osd pool application enable default.rgw.log rgw
ceph osd pool application enable default.rgw.meta rgw
ceph osd pool application enable default.rgw.buckets.data rgw
ceph osd pool application enable default.rgw.buckets.index rgw

# echo "[client.rgw.node1]
# host=node1
# keyring=/etc/ceph/keyring.radosgw.gateway
# log file=/var/log/radosgw/client.radosgw.gateway.log
# rgw frontends = civetweb port=8080
# " | sudo tee -a /etc/ceph/ceph.conf
# sudo ceph-authtool -C -n client.rgw.node1 --gen-key /etc/ceph/keyring.radosgw.gateway
# sudo chown $USER:$USER /etc/ceph/keyring.radosgw.gateway
# sudo ceph-authtool -n client.rgw.node1 --cap mon 'allow rw' --cap osd 'allow rwx' /etc/ceph/keyring.radosgw.gateway
# sudo ceph -k /etc/ceph/ceph.client.admin.keyring auth add client.rgw.node1 -i /etc/ceph/keyring.radosgw.gateway
# sudo radosgw -c /etc/ceph/ceph.conf -n client.rgw.node1 --keyring=/etc/ceph/keyring.radosgw.gateway

# copy conf/keyring
scp /etc/ceph/ceph.client.admin.keyring root@node2:/etc/ceph/
scp /etc/ceph/ceph.client.admin.keyring root@node3:/etc/ceph/
scp /etc/ceph/ceph.conf root@node2:/etc/ceph/
scp /etc/ceph/ceph.conf root@node3:/etc/ceph/
scp /etc/ceph/ceph.client.admin.keyring root@client:/etc/ceph/
scp /etc/ceph/ceph.conf root@client:/etc/ceph/
scp /etc/ceph/ceph.client.admin.keyring root@10.10.9.20:/etc/ceph/
scp /etc/ceph/ceph.conf root@10.10.9.20:/etc/ceph/
for i in {2..3}
do ssh node${i} "\
   sudo mkdir -p /var/lib/ceph/bootstrap-osd/
   sudo chown -R $USER:$USER /var/lib/ceph
   sudo chown -R $USER:$USER /etc/ceph
"
done
ssh client "sudo chown -R $USER:$USER /etc/ceph/"
scp /var/lib/ceph/bootstrap-osd/ceph.keyring root@node2:/var/lib/ceph/bootstrap-osd/ceph.keyring
scp /var/lib/ceph/bootstrap-osd/ceph.keyring root@node3:/var/lib/ceph/bootstrap-osd/ceph.keyring


# start osd in node2/3
for i in {2..3}
do
   ssh node${i} "\
   cd ~/ceph_fabric_config
   ./ceph_osd_deploy.sh"
done

# rgw
echo "[client] start ceph-rgw in client ..."
ssh client "
sudo mkdir -p /var/log/radosgw
sudo chown $USER:$USER /var/log/radosgw
echo '[client.rgw.client]
host=client
keyring=/etc/ceph/keyring.radosgw.gateway
log file=/var/log/radosgw/client.radosgw.gateway.log
rgw_frontends = civetweb port=8080
' | sudo tee -a /etc/ceph/ceph.conf
sudo ceph-authtool -C -n client.rgw.client --gen-key /etc/ceph/keyring.radosgw.gateway
sudo chown $USER:$USER /etc/ceph/keyring.radosgw.gateway
sudo ceph-authtool -n client.rgw.client --cap mon 'allow rw' --cap osd 'allow rwx' /etc/ceph/keyring.radosgw.gateway
sudo ceph -k /etc/ceph/ceph.client.admin.keyring auth add client.rgw.client -i /etc/ceph/keyring.radosgw.gateway
sudo radosgw -c /etc/ceph/ceph.conf -n client.rgw.client --keyring=/etc/ceph/keyring.radosgw.gateway
"
echo "[client] finish start ceph-rgw in client ..."

# create s3 API user & save secret key
# radosgw-admin user list
# radosgw-admin user rm --uid="testuser"
radosgw-admin user create --uid="testuser" --display-name="First User" > ~/ceph_fabric_config/secret.json

echo "finish deploy a single mon's cluster!"